'''
# -*- coding: utf-8 -*-
"""

TradeZone Client

Based on Example included in TradeZone Data Center (TZDC) installation.

Important decisions about data:
  - ParseTimestamp1() replaces time with 23:59 when the period is not Min1
  - ParseTimestamp1() and ParseTimestamp2() are trimming seconds to zero
  - ParseHistory() skips AfterMarket and Auction records that are NOT FROM TODAY.

To defeat urllib2's intrinsic buffering, you could do:
    import socket
    socket._fileobject.default_bufsize = 0
because it's actualy socket._fileobject that buffers underneath. No data will be
lost anyway, but with the default buffering (8192 bytes at a time) data may end up 
overly chunked for real-time streaming purposes (completely removing the buffering
might hurt performance, but you could try smaller chunks). [2]


References:
  [1] C# Example included in TradeZone Data Center (TZDC) installation.
  [2] http://stackoverflow.com/questions/1598331/how-to-read-continous-http-streaming-data-in-python

"""
# Realtime is simulated (random prices of PETR4 only) instead of the real streaming
FLAG_RT_SIM = False

import datetime
import time
#import dateutil.parser
import urllib2 as ul
import Pyro4
import string
import logging
import sys
import numpy as np
from threading import RLock

from misc.misc import enum
from misc import misc
import infogetter
from misc import timedata
import threads
import rtclasses
import misc.quoteperiod as qp

# Base socket and HTTPClient.__Get() timeout
# TODO: synchonize timeout with database server because it is specified there the Pyro commit timeout
TIMEOUT = 60
# Maximum number of tries for some operations that insist with TZDC
MAXTRIES = 7
import socket
socket.setdefaulttimeout(TIMEOUT) # Hopefully this will reflect in readlines() below
socket._fileobject.default_bufsize = 64 
# This option is useful for debugging. If the flag is set, the interface methods
# Will propagate exceptions
FLAG_RAISE_EXCEPTIONS = False

#############################################################################################################
## Realtime stream reader/poster

class Streamer(threads.LoopThread):
    """Reads stream from the TradeZone server and posts on the Pyro Event Server"""
    ## Setup
    # Reference time (in seconds) between posts to the Event Server
    cycleTime = 5
    # Number of characters to read from stream each time reading is attempted
    biteSize = 512
    # Simulation mode, this is just for debugging
    flagSim = FLAG_RT_SIM
    
    
    ############################################################################
    ## Interface routines
    
    def Pause(self, flag):
        TODO: any lock here?
        self._flagToPause = flag
        self._flagToResume = flag




    @misc.CatchException
    def SetSymbols(self, symbols):
        """Adds and/or removes symbols on TZDC to match the symbols parameter."""
        with self.R_rt:
            self._SetSymbols(symbols)
    
    def _SetSymbols(self, symbols):
        """Adds/Removes symbols from TZDC according to _symbols and updates it."""
        
        toAdd = set(symbols)-set(self._symbols)
        toRemove = set(self._symbols)-set(symbols)
        for s in toRemove:
            have to see who calls and recover from errors, maybe I don't want _Cycle() to blow
            self.tz._RemoveSymbol(s)
        for s in toAdd:
            self.tz._rt_AddSymbol(s)
        self._symbols = symbols
        
    def Get_symbols(self):
        with self.R_rt:
            return self._symbols


    
        
        
        """Starts realtime streaming."""
        with self.R_rt:
            
            
    ----        essa porra vai cagar, o streamer tem que poder regenerar outro loginId
            url = URLTemplates.GetRealtime.format(self.loginId)
            
    ----        nao tem que criar aqui, 
            self.logger.debug("We are about to attempt to start the streamer")
            self.streamer.Start()
            self.flagRealtime = True
            self.logger.debug("Realtime streaming started")
            
            
        
    @misc.CatchException
    def rt_Stop(self):
        with self.R_rt:
            if not self.flagRealtime:
                raise HTTPClientError("Realtime streaming not started!")
            else:
                self.logger.debug("StopRealTime(): stopping...")
                self.streamer.Exit()
                self.streamer.join()
                self.logger.debug("StopRealTime(): stopped...")
TODO: de repente esse clearsymbols pode ser na hora que sai fora, (BeforeExit, mas se pah foda-se)
                self.rt_SetSymbols([])


    ############################################################################
    ############################################################################
    
    ## State variables
    _urlFile = None
    _loner = ""
    _quotess = {}
    

    def __init__(self, url):
        threads.LoopThread.__init__(self, name="TZ Streamer")
        self.url = url
        O URLFile tem que ser aberto no _Cycle
        if not self.flagSim:
            self._urlFile = ul.urlopen(url, timeout=10)
        self.publisher = rtclasses.QuotePublisher()
        self._tic = -1 # Last time that quotes were published




    def _BeforeExit(self):
        if not self.flagSim:
            self._urlFile.close()
    
    def _Cycle(self):
        if self.flagSim:
            self.SimulatedCycle()
            return
            
        if self._tic != -1:
            timeLeft = self.cycleTime-(time.time()-self._tic)
            if timeLeft <= 0:
                # Publishes if time is due
                if not misc.isempty(self._quotess):
                    # Marks new time before publishing so that publishing time
                    # is discounted
                    self._tic = time.time()
                    self._PostQuotess()
        else:
            self._tic = time.time()

                
        try:
#            self.logger.debug("Going for reading %d bytes..." % self.biteSize)
            s = self._urlFile.read(self.biteSize)
#            self.logger.debug("What I read (%d bytes) was: %s" % (len(s), s.__str__()))
            
            s = self._loner+s # Joins with previous partial line
            self._loner = "" # Resets previous partial line
            bHas = True
            if s[-2:] != "\r\n":
                if not "\r\n" in s:
                    self._loner = s
                    bHas = False
#                    self.logger.debug("No full line, will read a bit more...")
                else:
                    splitPoint = s.rfind("\r\n")
                    self._loner = s[splitPoint+2:]
                    s = s[0:splitPoint]
            
            if bHas:
                self._quotess = ParseTicks(s.split("\r\n"), self._quotess)
        except socket.timeout:
            self.logger.debug("Timeout, nevermind...")
        except Exception as E:
            self.logger.exception("Error reading stream")
            self._PostError(E)
            if FLAG_RAISE_EXCEPTIONS:
                raise # Note that this will cause Streamer to exit
        
    def _PostQuotess(self):
        e = rtclasses.QuoteEvent()
        e.quotess = self._quotess
        self.logger.debug("PostQuotess(): About to publish quotes for %d securities" % (len(e.quotess),))
        i = 0
        flagOk = False
        while i < MAXTRIES:
            try:
                self.publisher.Publish(e)
                flagOk = True
                break
            except:
                self.logger.exception("PostQuotess(): Attempt %d/%d failed to publish quotes" % (i+1, MAXTRIES))
            i += 1
        if flagOk:
            # TODO: I have to think about a better error treatment, other than
            # just keep accumulating unpublished quotes
            
            self._quotess = {}
        
    def _PostError(self, E):
        """Read errors are posted through the event server."""
        e = rtclasses.QuoteEvent()
        e.flagError = True
        e.error = E
        self.logger.debug("PostError() about to publish")
        self.publisher.Publish(e)

    def _SimulatedCycle(self):
        def rand(range):
            return np.round(np.random.random()*range*100)/100
            
        # Figures out whether to wait or not
        if self._tic != -1:
            timeLeft = self.cycleTime-(time.time()-self._tic)
            self.logger.debug("Time left: %f" % timeLeft)
            if timeLeft > 0:
                self.logger.debug("Sleeping %f seconds" % timeLeft)
                time.sleep(timeLeft)
        self._tic = time.time()
        
        q = timedata.Quotes()
        q.timestamp = np.array([misc.dt2ts(
            datetime.datetime.today().replace(second=0, microsecond=0))])
        q.open = np.array([rand(10)+10])
        q.close = np.array([rand(10)+10])
        q.high = np.array([np.max([q.open, q.close])+rand(2)])
        q.low = np.array([np.min([q.open, q.close])-rand(2)])
        q.volume = np.array([rand(3000)+1000], dtype=float)
        q.numTrades = np.array(q.volume/100, dtype=int)
        q.status = np.array([1], dtype=int)
        d = {"PETR4": q}
        e = rtclasses.QuoteEvent()
        e.quotess = d
        self.logger.debug("PostSim() about to publish %s" % (e.quotess.__str__(),))
        self.publisher.Publish(e)



#############################################################################################################
## HTTP Client

def GetHTTPClientProxy():
    o = Pyro4.Proxy("PYRONAME:tradezone.HTTPClient")
    try:
        o.loginId
    except:
        raise ProxyError("Could not get a valid proxy to the HTTPClient object")
    return o

def GetStreamerProxy():
    o = Pyro4.Proxy("PYRONAME:tradezone.Streamer")
    try:
        o.loginId
    except:
        raise ProxyError("Could not get a valid proxy to the Streamer object")
    return o

class HTTPClient(infogetter.InfoGetter):
    """HTTP client for the TradeZone Data Center Server.
    
    This module is NOT thread-safe but is probably being accessed by at least
    two threads. Since it does not write to the database, it became minor
    concern. Besides, the communication with the TZ Data Center is already
    very unstable."""
    
    ## Setup variables
    # Number of tries of __Get() method.
    numTries = 7

    ## State variables
    loginId = ""
    isLogged = False
    # Whether realtime streaming is running 
    flagRealtime = False
    # Realtime symbols, maintained by rt_SetSymbols()
    symbols = []
    # Streamer object
    streamer = None
    
    ## Internal
    # symbols actually added, maintained by _rt_SetSymbols()
    _symbols = []

    
    def __init__(self):
        self.logger = logging.Logger(name="TZHTTPClient", level=logging.DEBUG)
        fmtr = logging.Formatter(fmt='%(levelname)s:%(message)s')
        ch = logging.FileHandler(filename="tradezone.log")
        ch.setFormatter(fmtr)
        self.logger.addHandler(ch)
        ch = logging.StreamHandler(sys.stdout)
        ch.setFormatter(fmtr)
        self.logger.addHandler(ch) # Logs both to file and console
        
        self.Ev_login = threading.Event() # This Event is managed by Login()
        # Events are initially set so that wait() calls don't block (initially)
        self.Ev_login.set() 
        # This Event is managed by _IncCounter()() and _DecCounter()
        self.Ev_counter = threading.Event() 
        self.Ev_counter.set()
        # This lock is used to make incrementing and decrementing the counter
        # thread-safe
        self.L_counter = threading.Lock()
        # Counts how many "other" methods are currently entered
        self._counter = 0

        # Lock to get/set _symbols etc
        R_rt = RLock() 


    def _IncCounter(self):
        with self.L_counter:
            self._counter += 1
            # This will keep Login() from entering
            self.Ev_counter.clear()
        
    def _DecCounter(self):
        with self.L_counter:
            self._counter -= 1
            if self._counter <= 0:
                # This will allow Login() from entering again
                self.Ev_counter.set()



                
    ###########################################################################
    ## These routines use __Get(), so they are controlled as the others below
        
    def _rt_AddSymbol(self, symbol):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.AddRealtime.format(self.loginId, symbol)
            i = 0
            flagOk = False
            while i < MAXTRIES: # Loop to insist on Realtime, not Delay
                # Example of reply: "PETR4=DL OK" <-- delay, not wanted, BTW
                a = self.__Get(url)
                if len(a) > 0:
                    s = a[0]
                    if not ("=RT" in s): # or "=DL" in s): #TODO temporary accepting delay
                        self.logger.warning("(try %d) rt_AddSymbol() got \"%s\"" % 
                            (i, s)) # same message, currently
                    else:
                        flagOk = True
                        break
                i += 1
            if not flagOk:
                raise RuntimeError("Couldn't add symbol %s, gave up after %d tries"%
                    (symbol, i))
            self.logger.debug("rt_AddSymbol(): success: %s" % symbol)
        finally:
            self._DecCounter()
        
    def _rt_RemoveSymbol(self, symbol):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.RemoveRealtime.format(self.loginId, symbol)
            s = self.__Get(url)
            self.logger.debug("rt_RemoveSymbol(): success: %s" % symbol)
        finally:
            self._DecCounter()

    ###########################################################################

    def AssertLogged(self):
        if self.isLogged is False:
            raise HTTPClientError("Not logged, needs to log in first")

    def GetLoginURL(self):
        return URLTemplates.Login.format(
            "A762C7B7-AB84-485B-8D09-4EB948572348", "2.0.0.0", "Navala")

    def __Get(self, url, timeout=TIMEOUT):
        """
        General HTTP GET.
        
        Returns a list containing the lines of what was returned by the HTTP
        server. Tries a number of times
        
        TODO: replace the try insistence by a recovery method in cases such as HTTP 403.
        
        """
        while True:
            try:
                url_file = ul.urlopen(url, timeout=timeout)
                break
            except ul.HTTPError as E:
                
                if E.code == 403 and not "login" in url:
                    self.logger.warning("Got error 403, will reattempt login")
                    self.Login()
                    raise # Still has to raise because loginID has changed
                else:
                    self.logger.critical("URL: "+url)
                    raise
            except:
                self.logger.critical("URL: "+url)
                raise
        # print(url_file.geturl())
        # print(url_file.info())
        i = 1
        flagOk = False
        while i <= self.numTries:
            try:
                a = url_file.readlines()
#                for aa in a:
#                    self.logger.debug(aa
                flagOk = True
            except Exception as E:
                i += 1
                if i > self.numTries:
                    self.logger.debug("Gave up after %d tr%s" % (i-1, "y" if i-1 == 1 else "ies"))
                    raise E
                self.logger.exception(E)
                self.logger.debug("Going for try %d/%d for url %s" % (i, self.numTries, url))
            if flagOk:
                break
        return a

    @misc.CatchException
    def Login(self):
        self.Ev_counter.wait() # Waits until al other methods have exited
        self.Ev_login.clear() # Blocks all other methods from entering again
        # Increases this counter even here because
        # I don't want Login() to be entered twice
        self._IncCounter()
        try:
            self.logger.debug("Logging...")
            url = self.GetLoginURL()
            a = self.__Get(url, timeout=15)
            if "LOCAL" in a[0]:
                raise RuntimeError("I don't want to log in locally")
            self.loginId = a[0]
            self.logger.info("LoginID=%s" % self.loginId)
            self.isLogged = True
            self.logger.info("Login(): success")
        finally:
            self.Ev_login.set()
            self._DecCounter()
        
    def GetLoginId(self):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            return self.loginId
        finally:
            self._DecCounter()
        
    @misc.CatchException
    def Logout(self):
        self.Ev_login.wait()
        self._IncCounter()
        try:
            self.logger.debug("Logging out...")
            self.AssertLogged()
            url = URLTemplates.Logout.format(self.loginId)
            self.__Get(url)
            self.loginId = ""
            self.isLogged = False
            self.logger.debug("Logout(): success")
        finally:
            self._DecCounter()
    
    @misc.CatchException
    def GetSymbols(self, pattern=""):
        """Gets list of symbols.
        
        pattern="" Parameter
        """
        self.Ev_login.wait()
        self._IncCounter()
        try:
            self.logger.debug("GetSymbols() start")
            url = URLTemplates.GetSymbols.format(self.loginId, pattern)
            a = self.__Get(url)
            result = ParseSymbols(a)
            self.logger.debug("GetSymbols() finish (%d symbols)" % len(result))
            return result
        finally:
            self._DecCounter()
                
    @misc.CatchException
    def GetHistory(self, symbol, from_=datetime.date(2013, 1, 16),
                   to=datetime.date(2013, 1, 16),
                   period=qp.min1):
        """
        Returns historical data as a timedata.Quotes object
        
        Arguments:
            symbol -- e.g. "PETR4"
            from_, to -- datetime.datetime, or string "YYYY-MM-DD HH:MM"
            period=quoteperiod.min1 -- values from quoteperiod module
        """
        self.Ev_login.wait()
        self._IncCounter()
        try:
            # Converts "from" to string, if necessary
            from_ = misc.dt2str(from_, False)
            to = misc.dt2str(misc.AddTimeIfZero(to), False)
            sPeriod = "1" if period == qp.min1 \
                      else qp.data[period]["english"]
            self.logger.debug("GetHistory() start: %s - %s to %s" % (symbol, from_, to))
            url = URLTemplates.GetHistory.format(self.loginId, symbol, from_,
                  to, sPeriod, "0") # "maxbars" is always zero
            a = self.__Get(url)
            # Result is reversed because TradeZone returns in decreasing time order
            #self.logger.debug("A HAS %d lines" % len(a))
            result = ParseHistory(a[-1::-1], period)
            s1 = misc.dt2str(misc.ts2dt(result.timestamp[0]), False) if len(result) > 0 else "?"
            s2 = misc.dt2str(misc.ts2dt(result.timestamp[-1]), False) if len(result) > 0 else "?"
            self.logger.debug("GetHistory() finish (%d quotes): %s to %s" % (len(result), s1, s2))
            return result
        finally:
            self._DecCounter()

    @misc.CatchException
    def AddMini(self, symbol):
        # TODO: parse
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.AddMini.format(self.loginId, symbol)
            s = self.__Get(url)
        finally:
            self._DecCounter()
        
    @misc.CatchException
    def RemoveMini(self, symbol):
        # TODO: parse
        self.Ev_login.wait()
        self._IncCounter()
        try:
            url = URLTemplates.RemoveMini.format(self.loginId, symbol)
            s = self.__Get(url)
        finally:
            self._DecCounter()


#############################################################################################################
# Types

# Tick type
TickType = enum('Full', 'History', 'TimeStamp')
# Minibook type
MinibookType = enum('Full', 'History', 'Compact')

class Tick(object):
    """Tick class
    """
    # string; Symbol name
    Stock = None
    # timestamp
    Timestamp = None
    # unsigned int
    PreviousOrderNumber = None
    # unsigned int    
    OrderNumber = None
    # unsigned int                        
    TradeVolume = None
    # unsigned int                        
    TotalVolume = None
    # float
    TradeFinancialVolume = None
    # float
    TotalFinancialVolume = None
    # float
    Open = None
    # float
    High = None
    # float
    Low = None
    # float
    PreviousClose = None
    # float
    Last = None
    # String
    Status = " ";
    # Possible values are within TickType
    Type = TickType.TimeStamp

class Minibook(object):
    """Minibook class
    """
    # string
    Stock = None
    # List of MinibookItem objects
    Items = []
    # string
    Status = None
    # Possible values within MinibookType
    Type = MinibookType.Full
    
class MinibookItem(object):
    """Minibook item: elements of Minibook.Items
    """
    # byte
    Order = None
    # ulong 
    TotalQuantityBids = None
    # uint
    NumberOfBids = None
    # float
    BestBidPrice = None
    # float
    BestAskPrice = None
    # uint
    NumberOfAsks = None
    # ulong
    TotalQuantityAsks = None

#############################################################################################################
# Parsers

def MakeFloat(s):
    return float(string.replace(s.strip(), ",", "."))

def MakeInt(s):
    return int(float(string.replace(s.strip(), ",", ".")))

def ParseTimestamp1(s, period):
    """Parses date-time string from "gethistory" into a datetime.datetime object."""
    dt = datetime.datetime.strptime(s.strip(), 
         "%d/%m/%Y %H:%M:%S") #.replace(second=0)
    if period >= qp.daily:
        dt = dt.replace(hour=23, minute=59, second=0)
    return (dt, misc.dt2ts(dt))

def ParseTimestamp2(s):
    """Parses date-time string from "getrealtime" into a datetime.datetime object."""
    dt = datetime.datetime.strptime(s.strip(), "%Y-%m-%d %H:%M:%S").replace(second=0) # TODO: Temporarily replacing second to see feature of updating that shit
    return misc.dt2ts(dt)

def ParseSymbols(rows):
    """
    Parses a string returned by the TradeZone server following the "getsymbols"
    command.
    
    """

    a = []
    for row in rows:
        if len(row) == 0:
            continue
        ff = row.split(";")
        oi = infogetter.SymbolsItem()
        oi.symbol = ff[0].strip()
        oi.description = ff[1].strip()
        oi.market = ff[2].strip()
        oi.flagDivide1000 = ff[3] == "True"
        oi.group = ff[4].strip()
        oi.supermarket = ff[5].strip()
        
        a.append(oi)

    return a

def ParseHistory(rows, period):
    """
    Parses rows returned by the TradeZone server following the "gethistory"
    command.
    
    period influences time rounding: non-intraday ticks should be given
    always with the same time (See ParseTimestamp1())
    """
    # Símbolo ; Nome ; Mercado ; Divide por mil ; Grupo ; Bolsa CR/LF

    dToday = datetime.datetime.today().date()
    numRows = len(rows)
    a = timedata.Quotes()
    a.SetSize(numRows) # Hope that setting size, then eventually shrinking is faster
                       # than appending.
    cnt = 0 # Number of rows actually added
    for row in rows:
        if len(row) == 0:
            continue
        ff = row.split(";")
        # Parses date+time and status first to decide whether will skip or not
        (dt, ts) = ParseTimestamp1(ff[0], period)
        if cnt > 0 and ts == a.timestamp[cnt-1]:
            # Tradezone is sending me some repeated quotes for some reason
            continue
        status = timedata.QuoteStatus.Regular
        temp = ff[8].strip()
        if temp == "A":
            status = timedata.QuoteStatus.AfterMarket
        elif temp == "L":
            status = timedata.QuoteStatus.Auction
        if status != timedata.QuoteStatus.Regular and dt.date() != dToday:
            continue

        a.timestamp[cnt] = ts
        a.open[cnt] = MakeFloat(ff[1])
        a.high[cnt] = MakeFloat(ff[2])
        a.low[cnt] = MakeFloat(ff[3])
        a.close[cnt] = MakeFloat(ff[4])
        #a.Qtt = MakeInt(ff[5])
        a.volume[cnt] = MakeInt(ff[6])
#####        print("MERDA --> %s; %.2g; %.2g; %.2g; %.2g; %d" % (misc.ts2dt(ts), a.open[cnt], 
#####            a.high[cnt], a.low[cnt], a.close[cnt], a.volume[cnt]))
        a.numTrades[cnt] = MakeInt(ff[7])
        a.status[cnt] = status
        a.flagRealtime[cnt] = False
        cnt += 1
    if cnt < numRows:
        a.SetSize(cnt)
    return a

def ParseMinibook(s):
    """Returns a Minibook object
    """
#    CultureInfo ci = new CultureInfo("pt-BR", false);
    mt = MinibookType.History
    if s.startswith("HM"):
        mt = MinibookType.History
    elif s.startswith("FM"):
        mt = MinibookType.Full
    elif s.startswith("CM"):
        mt = MinibookType.Compact
    else:
        return None

    s = s[3:]
    cols = s.Split(';')

    m = Minibook()
#    m.Items = new List<MinibookItem>()
    m.Type = mt
    m.Stock = cols[0].strip()
    m.Status = cols[1].strip()

    for i in range(2, len(cols)):
        s1 = cols[i].Split('=');
        if (len(s1) != 2):
            continue
        order = s1[0]
        fields = s1[1][1:-3].split(':')
        mi = MinibookItem()
        mi.order = MakeInt(order)

        mi.BestAskPrice = MakeFloat(fields[0])
        mi.NumberOfAsks = MakeInt(fields[1])
        mi.TotalQuantityAsks = MakeInt(fields[2])
        mi.BestBidPrice = MakeFloat(fields[3])
        mi.NumberOfBids = MakeInt(fields[4])
        mi.TotalQuantityBids = MakeInt(fields[5])
        
        m.Items.append(mi)

#    m.Items.Sort(delegate(MinibookItem m1, MinibookItem m2) { return m1.order.CompareTo(m2.order); });
    return m




def ParseTicks(rows, a):
    """Returns a dictionary containing symbols as keys and timedata.Quotes as values
    
    Arguments:
      rows -- result of a "getrealtime" command
      a -- already existing dictionary to append information
        
    Returns:
      {"symbol": timedate.Quotes, ...}
      Note that the symbol is just "PETR4" or "VALE5", and not "BOVESPA.VISTA.PETR4"
            
    Full tick example:
      0 FT=PETR4;      Full Tick = Symbol
      1 2013-03-06;    date
      2 17:53:07;      time
      3 16,56;         yesterday's close
      4 17,6;          today's open
      5 17,69;         today's maximum
      6 17,22;         today's minimum
      7 17,66;         today's last value
      8 100;           volume (number of papers) (probably of last trade)
      9 46292400;      total volume 
      10 1766;         volume in money (probably of last trade)
      11 8,104275E+08; total volume in money (I suppose)
      12 48664;        previous record id
      13 48665;           current record id
    """
    for s in rows:
        if len(s) <= 2:
            continue # Empty line
        type_ = s[0:2]
        if type_ in ("TS", "HT"):
            # timestamp and history ticks. Valid type_ is "FT" (Full Tick)
            continue

        s = s[3:]
        cols = s.split(';')

        stock = cols[0].strip()
        if not a.has_key(stock):
            q = timedata.Quotes()
            a[stock] = q
        else:
            q = a[stock]

        ts = ParseTimestamp2(cols[1].strip()+" "+cols[2].strip())
        last, numTrades = map(lambda x: MakeFloat(cols[x]), [7, 8])
        volume = MakeInt(cols[10])
        i = misc.BSearch(q.timestamp, ts)
        if i == -1:
            # New quote
            i = len(q)
            q.SetSize(i+1)
            q.timestamp[i] = ts
            q.open[i] = last
            q.close[i] = last
            q.high[i] = last
            q.low[i] = last
            q.numTrades[i] = numTrades
            q.volume[i] = volume
            q.flagRealtime[i] = True # TODO: temporary
            status = timedata.QuoteStatus.Regular        
            if len(cols) >= 15:
                temp = cols[14].strip()
                if temp == "A":
                    status = timedata.QuoteStatus.AfterMarket
                elif temp == "L":
                    status = timedata.QuoteStatus.Auction
            q.status[i] = status
        else:
            # Quote update
            q.close[i] = last
            q.high[i] = max(q.high[i], last)
            q.low[i] = min(q.low[i], last)
            q.numTrades[i] += numTrades
            q.volume[i] += volume
    return a



#############################################################################################################
# URL Templates

class URLTemplates(object):
    """Templates for the HTTP GET strings to communicate with the TZDataCenter
    """
    Server = "http://localhost:31008"

    Login = Server + "/login?product={0}&version={1}&description={2}";
    Logout = Server + "/logout?id={0}";

    GetSymbols = Server+ "/getsymbols?id={0}&pattern={1}";
    GetHistory = Server+ "/gethistory?id={0}&symbol={1}&from={2}&to={3}&size={4}&maxbars={5}";
    GetRealtime = Server+ "/getrealtime?id={0}";

    AddRealtime = Server + "/rt_order_symb?id={0}&symbol={1}";
    RemoveRealtime = Server + "/rt_order_symb_remove?id={0}&symbol={1}";

    AddMini = Server + "/rt_mini_symb?id={0}&symbol={1}";
    RemoveMini = Server + "/rt_mini_symb_remove?id={0}&symbol={1}";

#############################################################################################################
# Errors
class HTTPClientError(Exception):
    pass

class ProxyError(Exception):
    pass

'''
